import pymysql,datetime,re
from selenium import webdriver
from crawlab import save_item
url = "http://www.kifa.net.cn/info.do?method=pageQuery&type=%E8%A1%8C%E6%83%85%E5%88%86%E6%9E%90"
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
options.add_argument('--disable-gpu')
options.add_argument('--disable-dev-shm-usage')
options.add_experimental_option('excludeSwitches', ['enable-automation'])
settings = { 'host':'172.16.10.187','user':'tidb','port':3306,'password':'Tidb@guest@123','database':'crawl','charset' : 'utf8'}
prefs = {
    'profile.default_content_setting_values': {
        'images': 2,
    }
}
options.add_argument('log-level=3')
# options.add_experimental_option("prefs", prefs)
options.headless = False # 开启无头模式
driver = webdriver.Chrome(options=options)
conn = pymysql.connect(**settings)

sql = '''insert into flower_trading(url,title,time,trading_volume_total,'''\
                        '''trading_volume_curl,supply_position,supply_num,market,closing,'''\
                        '''max_type,min_type,max_price,ups_donws,ups_donws_num,unit_price,'''\
                        '''actual_price,output,ave_unitl_price,create_time )values '''\
                        '''(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''

cursor=conn.cursor()


if __name__ == '__main__':
    driver.get(url)
    current_list = driver.find_elements_by_xpath('/html/body/div[3]/div[2]/div/div/div/a')
    path_list = []
    for i in current_list:
        m = i.text.index('月')
        d = i.text.index('日')
        day = int(i.text[m+1:d])
        now = datetime.datetime.now()
        #  只爬取当天的
        if day == int(now.strftime('%d')):
            path_list.append(i.get_attribute('href'))
    for j in path_list:
        j = 'http://www.kifa.net.cn/info.do?method=show&contentId=133942'
        driver.get(j)
        text_list = driver.find_element_by_xpath('/html/body/div[3]/div[2]/div[1]').text.strip().replace(' ','')
        text_list = text_list.replace('，',',').replace('。',',').split('\n')

        path =j
        role_index = 0
        babysbreath_index = 0
        mum_index = 0
        for i in text_list:
            if '玫瑰行情' in i:
                role_index =  text_list.index(i)
            if '满天星行情' in i:
                babysbreath_index =  text_list.index(i)
            # if '非洲菊行情' in i:
            #     mum_index =  text_list.index(i)
        a = text_list[role_index:babysbreath_index]
        b = text_list[babysbreath_index:mum_index]
        c = text_list[mum_index:]
        dicts = {}
        dicts['url'] = path
        dicts['title'] = text_list[0]
        dicts['time'] = text_list[1]
        try:
            dicts['trading_volume_total'] = re.findall('\d+', text_list[2].split(',')[1])[0]
            dicts['trading_volume_curl'] = re.findall('\d+', text_list[2].split(',')[2])[0]
        except:
            dicts['trading_volume_total'] = re.findall('\d+', text_list[3].split(',')[1])[0]
            dicts['trading_volume_curl'] = re.findall('\d+', text_list[3].split(',')[2])[0]
      
        for i in a[1].split(','):
            if '供货量较前一交易日' in i:
                dicts['max_type'] = i[:4]
                dicts['supply_position'] = i[-4:]
            if '交易行情' in i:
                dicts['market'] = i[-2:]
            if '成交率' in i:
                dicts['closing'] = i[3:]
            dicts['create_time'] = datetime.datetime.now()

        for i in a[2:6]:
            dic = dicts
            dics = {}
            index = a[2:6].index(i)
            if index>2:
                dics['max_type'] = '多头玫瑰'
                dics['market'] = i.split(',')[0].split('交易行情')[1]
                dics['closing'] = i.split(',')[1].split('成交率')[1]
                dics['max_price'] = i.split(',')[2].split('最高价')[1]
                dics['ave_unitl_price'] = i.split(',')[3].split('等级均价为')[1]
                c = {**dic,**dics}
                # save_item(c)
                row = (c.get('url'),c.get('title'),c.get('time'),c.get('trading_volume_total'),c.get('trading_volume_curl'),c.get('supply_position'),c.get('supply_num'),
                       c.get('market'),c.get('closing'),c.get('max_type'),c.get('min_type'),c.get('max_price'),c.get('ups_donws'),
                       c.get('ups_donws_num'),c.get('unit_price'),c.get('actual_price'),c.get('output'),c.get('ave_unitl_price'),c.get('create_time'))
                cursor.execute(sql,row)
                print(row)
                conn.commit()
            else:
                dics['min_type'] = i[:2]
                dics['max_price'] = i.split(',')[0].split('最高价')[1]
                d = re.findall('\d+(?:\.\d+)?',i.split(',')[2])[0]
                dics['ups_donws_num'] = d
                dics['ups_donws'] = i.split(',')[2].split(d)[0]
                c = {**dic, **dics}
                # save_item(c)
                row = (c.get('url'),c.get('title'), c.get('time'), c.get('trading_volume_total'), c.get('trading_volume_curl'),
                       c.get('supply_position'), c.get('supply_num'),
                       c.get('market'), c.get('closing'), c.get('max_type'), c.get('min_type'), c.get('max_price'),
                       c.get('ups_donws'),
                       c.get('ups_donws_num'), c.get('unit_price'), c.get('actual_price'), c.get('output'),
                       c.get('ave_unitl_price'), c.get('create_time'))
                cursor.execute(sql, row)
                print(row)
                conn.commit()


    conn.close()
    cursor.close()
    driver.close()






         
